{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "aSeI4R7URNY7",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 437.0
    },
    "outputId": "5d7620db-36f1-48af-cd82-6dbc7abb04c5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1000 cost = 2.294210\n",
      "Epoch: 2000 cost = 2.031863\n",
      "Epoch: 3000 cost = 1.944698\n",
      "Epoch: 4000 cost = 1.989209\n",
      "Epoch: 5000 cost = 1.808382\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd8AAAFKCAYAAABcq1WoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl4E3X+B/B37rb0TJvKUS/ALoeg\nZVFEQKAUrIuLXKUc4q66KCgogqIcbkEuAcVdKQi2CAWE1gJyquUo9yGyRS6PgoggILQ0SZMeSXP8\n/ujPSCVQWtJJZvJ+PY/Pw8w3yXw+zCPvzHcmMzKn0+kEERERCUbu7QKIiIj8DcOXiIhIYAxfIiIi\ngTF8iYiIBMbwJSIiEhjDl4iISGBKoTZUUGASalMeExERBL2+1NtleIyU+mEvvktK/UipF0Ba/Yih\nF50u5IZjPPK9CaVS4e0SPEpK/bAX3yWlfqTUCyCtfsTeC8OXyIcsXJiKdetWe7sMIqpjgk07E1H1\nhg8f6e0SiEgADF+iWrh06SKGD38WAwYMxqZN6+F0ApMmTUFGRjpOncrHww8/ggkTUpCbuw1LlnwM\nu92OqCgd3nxzEqxWK1566V/YuHELlMrK/wXHjx+Ldu3a4+TJE2jUKAb//Oe/8PPPZ/D++++isLAQ\narUKEyakoFmzFl7unIg8gdPORLVkMBig1UZi1aq1aNq0KVJSxmPixCnIyFiFbdtycOTI/zB79jTM\nnPk+Vq5cg/btO2L27Bm4997GiIyMxLFj3wIAysrK8L//HUbnzt1cn+1wODB+/OtITPwbMjPX4vXX\nx+Ott8bCZrN5q10i8iCGL1Et2e12xMcnAAAaN26K5s1bIDw8HGFh4YiMjMKRI/9DXFxbxMTcCQD4\n+99748iRw7DZbOjSpRv27t0FANizZw+aN2+JiIgI12f/8stZGAxF6NnzKQBA69YPIjw8AidOHBO4\nSyKqC5x2JqolhUIBjSYAACCXyxEYGOQak8vlUKnUCAn546cGwcHBcDqdMBoN6NKlGyZMeB2vvDIW\n27ZtQ7du3at8ttlsQnl5OYYM6e9aV1JSAqPRWMddEZEQGL5EdUSr1eLkyT+OVIuLiyGXy11HxnK5\nAqdO5WPv3r144YVRVd4bFaVDvXr1sHLlGqHLJiIBcNqZqI4olUp8++0RXLjwKwBg/fo1eOihdq6L\nrLp27YZPPvkYzZs3R1hYeJX31q/fADrdHdixYxuAyvPLKSkTUFZWJmwTRFQneORLVEd0umi89dYk\njB9feaFUgwaNMG7cBNd4ly7d8PzzT2PatGnXvVcmk2HKlBmYM2cG0tI+glwuR3LyEAQGBgrZAhHV\nEZnT6XQKsSEx3l5SpwsRZd03IqV+2IvvklI/UuoFkFY/YuiFt5ckEiGHowIVliI4HBXeLoWIPIzT\nzkQ+xul0QH9hC8oMP8JeYYRCFYbA8L8golEPyGT8vkwkBbcVvrNnz8b//vc/2Gw2vPjii+jRo4en\n6iLyW/oLW2AuOORatlcYXcvamERvlUVEHlTr8D148CBOnTqFrKws6PV69OnTh+FLdJscjgqUGX50\nO1ZmyIejYTfI5SqBqyIiT6t1+D700ENo3bo1ACA0NBRlZWWw2+1QKMT9mCcib7JXmGCvcH8jDXuF\nEfYKE+QarcBVEZGn1foEkkKhQFBQ5R19Vq9ejccee4zBS3SbFKoQKFRhNxgLg0J146sniUg8bvun\nRtu2bcOiRYvwySefVLmV3p/ZbHbRP/yYSAjnf1iPK+f2Xrc++q6OuLPZU16oiIg87bYuuNqzZw8W\nLlyI9PT0mwYvAOj1pbezKa8Qw+/IakJK/Ui5F422C4LLrCgz5F9ztXMsNNououhZyvtG7KTUjxh6\nudnvfGsdviaTCbNnz8bSpUsRHh5e/RuI6JbIZHJoYxLhaNgN9goTFKoQXmRFJDG1Dt8vvvgCer0e\no0ePdq2bNWsWGjZs6JHCiPydXK7ixVVEElXr8E1OTkZycrInayEiIvILvF0OERGRwBi+REREAmP4\nEhERCYzhS0REJDCGLxERkcAYvkREJAnvvPM2+vbtiY4d2+LHH3+44esuXbqIzp3bCVjZ9fg8XyIi\nkoRt23KwatVaNGoU4+1SqsXwJSIi0Rs58gU4HA6MGTMSJSUlmD59Dlq2vB/vvTcTR48egcPhQJMm\n92HixBTXezZtWo/s7FUwmUwYMWIUuncX7nnZnHYmIiLRS039GAAwb94iBAQEAAAOHTqIS5cuYuXK\nNcjM/Bz33tsYJ04cBwA4HA7YbBXIyMjEqFGvIS3tI0HrZfgSEZEkhYeH4+zZM9i9ewfKy8sxbNgI\ntGvXHgDgdDqRmPgkACA2thkKCq4IWhvDl4iIJKlFi/sxevQbWL06C716PY7JkyfCZKp8EpJCoXAd\nIcvlcjgcDkFrY/gSEZFkde2agHnzFmHNmo2wWMqxcuUyb5cEgOFLREQStXnzBixdmg4ACA0Nw113\n3QOZTOblqioxfImISJI6deqMH3/8HgMH9sGQIf1x9uwZDBw4xNtlAQBkTqfTKcSGCgpMQmzGo3S6\nEFHWfSNS6oe9+C4p9SOlXgBp9SOGXnS6kBuO8ciXiIj8kqXCjiv6Ulgq7IJvmzfZICIiv2J3OJCV\nexpH8gtQVGyBNlSDuFgdkuObQiEX5piU4UtERH4lK/c0th3+1bV8tdjiWh6cECtIDZx2JiIiv2Gp\nsONIfoHbsSP5hYJNQTN8iYjIbxjNFhQVW9yO6U3lMJrdj3kaw5eIiPxGWLAG2lCN27GIkACEBbsf\n8zSGLxER+Q2NSoG4WJ3bsbjYKGhUCkHq4AVXRETkV5LjmwKoPMerN5UjIiQAcbFRrvVCYPgSEZFf\nUcjlGJwQi36dm8BotiAsWCPYEe/vGL5EROSXNCoFoiOCvLJtnvMlIiISGMOXiIhIYAxfIiIigTF8\niYiIBMbwJSIiEhjDl4iISGAMXyIiIoExfImIiATG8CUiIhIYw5eIiEhgDF8iIiKBMXyJiIgExvAl\nIiISGMOXiIhIYAxfIiIigTF8iYiIBMbwJSIiEhjDl4iISGAMXyIiIoExfImIiATG8CUiIhIYw5eI\niEhgDF8iIiKBMXyJiIgExvAlIiISGMOXiIhIYAxfIiIigTF8iYiIBMbwJSIiEhjDl4iISGAMXyIi\nIoHdVvjm5+cjISEBK1as8FQ9REREklfr8C0tLcXUqVPRvn17T9ZDREQkebUOX7VajbS0NERHR3uy\nHiIiIslT1vqNSiWUylt/e0REEJRKRW035zU6XYi3S/AoKfXDXnyXlPqRUi+AtPoRcy+1Dt+a0utL\nhdqUx+h0ISgoMHm7DI+RUj/sxXdJqR8p9QJIqx8x9HKzLwe82pmIiEhgDF8iIiKB1Xra+cSJE5g1\naxYuXLgApVKJnJwczJs3D+Hh4Z6sj4iISHJqHb73338/li9f7slaiIiI/AKnnYmIiATG8CUSUF7e\nYSQn9/bIZ33xxUa8+upLHvksIhIWw5eIiEhggv3Ol4j+kJr6H+zduxtyuQyzZr2L6Oi78OGH7yMv\n7zDkcjkeeaQDXnrpFSgUCpw+fQrvvz8TRqMRarUGI0aMQrt2VW/rWlJixvDhz+G5515A164JXuqK\niG4Vw5dIYL/9dgnNmjXHyJGjsWHD53jnnXfw2GPdcOXKZSxf/hnsdhtGjnwB27bloHv3REyePAH/\n+Mfz6N49ET/88B1ee20k1qzZ6Po8h8OBKVMmoUePJxi8RCLBaWciganVasTHdwcAxMd3x/fff49d\nu3LRq1cfKJVKaDQB6N79CRw6dBCXLl3E1atXkZDwOACgWbMWqF+/Pr7//jvX5y1cmIrw8AgMHfqs\nV/ohoppj+BIJLDQ0DHJ55f969erVA1A5bRwSEup6TUhICPR6PfR6PYKDQyCTya4ZC4VeXwQA+PHH\n77BmTRZ0Oj7ghEhMGL5EAjOZTNf9OTQ0DEaj0bW+uNgIrVYLrVYLk8kIp9PpGjMajdBqIwEAUVE6\nLFuWhQ0bPsf3358UqAMiul0MXyKBWSzl2LVrBwBg587taNWqFTp37orNm9fDbrejrKwMOTlfoH37\njmjQoCF0umhs374FAHD8+FEUFV1F8+YtAQCRkTo0ahSDUaNew/Tpk2GxWLzWFxHdOl5wRSSwu+66\nGydPHsOiRamQy+WYM2c2tNqGuHjxAoYOHQCZTIauXRMQH58AmUyGKVNmYM6cmViyJA0BAYGYOvVd\nBAYGVvnMHj2ewM6dufj44wUYNeo1L3VGRLdK5rx2PqsO+fqjn9wRwyOrakJK/bAX3yWlfqTUCyCt\nfsTQCx8pSCRBlgo7ruhLYamwe7sUIqohTjsTiYzd4UBW7mkcyS9AUbEF2lAN4mJ1SI5vCoWc36eJ\nxIDhSyQyWbmnse3wr67lq8UW1/LghFhvlUVENcCvyUQiYqmw40h+gduxI/mFnIImEgmGL5GIGM0W\nFBW7/zmR3lQOo5k/NSISA4YvkYiEBWugDdW4HYsICUBYsPsxIvItDF8iEdGoFIiL1bkdi4uNgkal\nELgi8nW/P0N64cJUrFq1CgDQsWNbXLly2cuV+TdecEUkMsnxTQFUnuPVm8oRERKAuNgo13oid4YP\nHymK38b6C4Yvkcgo5HIMTohFv85NYDRbEBas4REvVWv69MmIjW2CpKShVdYvWjQfZ8+ewfTpc/DL\nL2fx/vvvorCwEGq1ChMmpKBZsxZeqljaGL5EIqVRKRAdEeTtMkjEtm/fgq+/PoD589MAAOPHv46n\nn34GTz7ZG8eOfYu33hqL1as3QqlkVHgaz/kSEfmh/PwfkJa2ELNmzUVgYCB++eUsDIYi9Oz5FACg\ndesHER4egRMnjnm5Umni1xkiIj80Z85MBAUFITS08jnSZrMJ5eXlGDKkv+s1JSUlVR51SZ7D8CUi\n8kMpKdOwfv1afPRRKkaPfh1RUTrUq1cPK1eu8XZpfoHTzkREfigm5k6MGTMOO3ZsQ17eYdSv3wA6\n3R3YsWMbAMBgMCAlZQLKysq8XKk0MXyJiPxUWFg43nhjPGbOfAdlZaWYMmUG1qz5DIMH98PIkcPQ\ntu3D1z07mjyDz/O9Can9Jk5K/bAX3yWlfqTUCyCtfsTQC5/nS0RE1bLarSgovQqr3ertUiSPF1wR\nEfk5u8OOtac341jBSegtBkRowtFa1xJ9m/aEQs4buNQFhi8RkZ9be3ozdv6617VcZNG7lpNie3mr\nLEnjtDMRkR+z2q04VnDS7djxwpOcgq4jDF8iIj9mtJigtxjcjhWVG2C0+PZFTWLF8CUi8mNhmhBE\naMLdjmkDwhGmufEVu1R7DF8iIj+mVqjRWtfS7VirqJZQK9QCV+QfeMEVEZGf69u0J4DKc7xF5QZo\nA8LRKqqlaz15HsOXiMjPKeQKJMX2wlNNEmG0mBCmCeERbx1j+BIREYDKKWhdUKS3y/ALPOdLREQk\nMIYvERGRwBi+REREAmP4EhERCYzhS0REJDCGLxERkcAYvkRERAJj+BIREQmM4UtERCQwhi8REZHA\nGL5EREQCY/gSEREJjOFLREQkMIYvERGRwBi+REREAmP4EhERCYzhS0REJDCGLxERkcCUtX3jjBkz\ncPToUchkMkyYMAGtW7f2ZF1ERESSVavwPXToEH755RdkZWXhp59+woQJE5CVleXp2oiIiCSpVtPO\nBw4cQEJCAgCgSZMmMBqNMJvNHi2MiIhIqmoVvoWFhYiIiHAta7VaFBQUeKwoIiIiKav1Od9rOZ3O\nal8TEREEpVLhic0JSqcL8XYJHiWlftiL75JSP1LqBZBWP2LupVbhGx0djcLCQtfylStXoNPpbvoe\nvb60NpvyKp0uBAUFJm+X4TFS6oe9+C4p9SOlXgBp9SOGXm725aBW084dOnRATk4OAODkyZOIjo5G\ncHBw7aojIiLyM7U68m3Tpg1atmyJgQMHQiaTISUlxdN1ERERSVatz/m+/vrrnqyDiIjIb/AOV0RE\nRAJj+BIREQmM4UtERCQwhi8REZHAGL5EREQCY/gSEREJjOFLREQkMIYvERGRwBi+REREAmP4EhER\nCYzhS0REJDCGLxERkcAYvkRERAJj+BIRScClSxfRuXO7Gr3n3Lmz+PbbvDqqiG6G4UtE5Kd27drJ\n8PWSWj/Pl4iIfM+mTeuRnb0KJpMJI0aMQrduPfDBB3Nw+PDXcDjsuP/+1hg/PgUHD+7HihVLoFKp\nYDKZMGrUa1i/fi2ysj6F1WpFy5atMGHCv6HRBHi7JUli+BIRSYTD4YDNVoGMjEzs2LENH300DyqV\nCseOHcHy5Z9Bqw1Cr15PYfv2LXj88b/hsce6olGjGPzzn//C0aNHkJ6+EEuWfIqoKB3mzJmBtLSF\nGDlytLfbkiROOxMRSYTT6URi4pMAgNjYZigouIIuXbohPX05lEolNBoNmjVrgYsXL1z33n37dqNb\nt+6IitIBAHr37oddu3YIWr8/4ZEvEZFEKBQKBARUThPL5XI4HA7o9Xr85z+z8eOPP0KlUqCgoABJ\nSYOue6/JZMaePTtw6NBBAIDD4YTNViFo/f6E4UtEJGEff7wASqUSy5ZlolGjSIwc+arb10VFRSEx\n8UlOMwuE085ERBJmMBShceOmUKvV+OGHH3D8+FGUlZUBAJRKJcxmMwCgY8fO2LVrB/R6PQBgz56d\nWLFiqbfKljwe+RIRSdjAgU9j2rTJ+OKLjWjX7mGMHDka7747FS1a3I8OHTphypRJ+O23i5g2bTae\neeZZjBr1IpxOByIitHjjjQneLl+yZE6n0ynEhgoKTEJsxqN0uhBR1n0jUuqHvfguKfUjpV4AafUj\nhl50upAbjnHamYiIAAAORwUqLEVwOHihVV1j+BIReVBe3mEkJ/f2dhk14nQ6UPTrV7j03QJc+i4V\nl75bgKJfv4LT6fB2aZLFc75ERH5Of2ELzAWHXMv2CqNrWRuT6K2yJI3hS0QEYOPGdcjMXAG73Y7I\nyCi8/fY7yMxcip9+Oguj0YCffjqN6OhozJjxHiIitOjf/+/o06c/tm/fiitXfsNTT/XDsGEjqnym\n1WrFggX/xcGDB2CzVaBXrz545pnnvNShew5HBcoMP7odKzPkw9GwG+RylcBVSR+nnYnI7+n1Rfjg\ng9n44IP5yMz8HI0axWDp0nQAwK5dOzB69BtYs2YTGjZshOXLl7jed+LEcaSlZWD58s+wdm02Tp3K\nr/K5K1cuw88//4xlyzKxfPln2LlzO/bt2yNob9WxV5hgrzDeYMwIe4VvX9QkVn4TvgsXpmLdutXe\nLoOIfFBEhBY5ObsQHX0HAOCBB+Jct2Bs0+avaNiwEQDgscficfz4Mdf7EhN7QqFQICJCiwceeBAn\nThyr8rn79u1G3779oVarERgYiMTEnti1K1egrm6NQhUChSrsBmNhUKhufMUu1Z7fTDsPHz7S2yUQ\nkY+y2+1IT1+Ifft2w263o7S0FHfeeRcAIDT0j2AKCQmByfTHkWBoaOg1Y6EwmYqrfK7JZMaHH87F\nokXzAQAVFRVo3rxlXbZSY3K5CoHhf6lyzvd3geGxnHKuIz4XvpcuXcTw4c9iwIDB2LRpPZxOYNKk\nKcjISMepU/l4+OFHkJjYE7NmTUNW1joAlVcX/r585sxpzJo1HSUlJbDZKpCUNBD9+iVj+vTJrqd3\n/PDD95g9ezpKS0sRFRWFCRNSXN9sicj/bN++Ffv27UZqahrCw8OxYcPn2LLlSwCAwWBwvc5kKq4S\nuEbjH2PFxcYqY0DlLRsHDRqKDh061XEHtyeiUQ8Aled47RVGKFRhCAyPda0nz/PJaWeDwQCtNhKr\nVq1F06ZNkZIyHhMnTkFGxips25aDixd/veF7P/kkDb1798OKFZ9h4cIlOHz4EKxWa5XXTJ48AcOG\njUBm5lo89lgXfPDB7LpuiYh8mMFQhPr1GyA8PBxGowG5uVtdt2A8duxbXL78GwBgx47teOCBONf7\ntm/fCofDgaKiqzh27Chat46r8rmdOnXGpk3rYLfb4XQ6sXRpOg4e3C9cY7dIJpNDG5OIBi1GoEGL\nl9GgxQhoYxIhk/lkREiCzx35ApVTQPHxCQCAxo2bAgDCw8MBAJGRUSgsLLzheyMitNi5czsaN26K\n2Ni/YObM96uMnzv3CwwGA9q37wAA6Nt3AJ56ql9dtEFEIpGQ8Di2bs1BcnJvNGzYCMOGvYS33hqD\nkpISPPRQO8ydOwunTuXjjjvqY/ToN1zvu/fexhg27B/47beL6N8/GY0bN0Fe3mHXeN++A3Dp0iUM\nHToATqcTzZq1wIABg73R4i2Ry1WQa7TeLsMv+GT4KhQKaDR/PBYrMDDINSaXy9G69YP48stNbt87\nYsQoLF++BP/+91uwWq0YOvRZ9O2b5Bo3Gg0IDg52LSuVSiiVPvnXQEQC0WojkZaWUWXdxo1bkJm5\nFPXq1cPUqe+6fd9f//ownnvuhSrr2rRp6zolplKpMHr063VTNImaKOcUjh37Fg7HH3deufYCiKCg\nILz44svIylqHGTPmID19Ic6d+8U1HhYWjuLiYtf7bTYbLl26KFzxROQ3LBV2XNGXwlJh93Yp5GNE\nGb6RkVG4erUQen0R7HY7tm790jU2btxrOHPmJwBA48ZNUK9eMGQymWv8zjvvQnR0tOty/02b1mH2\n7OnCNkBEkmZ3OLByWz4mpR3E+EUHMSntIFZuy4fdwds11pbVar3hjKcYiXK+tVGjGPztb73w7LND\ncMcd9ZGY2NP14/b+/ZMxZcok2GyVNwbv06e/6ycDACCTyfDOO+9i6tS3sWjRfERGVl7tTET0Z6NG\njbrhk3NWr954w/dl5Z7GtsN/XBh6tdjiWh6cEOvZIv1Efv6P+OqrL/DEE096uxSP4CMFb0IMj6yq\nCSn1w158l5T6qU0vlgo7JqUdxNViy3VjkaEBmDasHTQqhadKrBFf2zdffrkJGRmfAABatmyJN998\nGzk5X1x3m0+1Wo1nnx2MkpISxMY2w4IF6T7Xizt8pOCfWO1WFJRehdVurf7FREQ1YDRbUOQmeAFA\nbyqH0ex+zN9cunQR8+f/F6mpi7Bq1RqUlZXjs89Wur3Np1YbiRdfHImWLVtjwYJ0b5fuEaKcdq4t\nu8OOtac341jBSegtBkRowtFa1xJ9m/aEQu6db6JEJC1hwRpoQzVuj3wjQgIQFqzxQlW+59Chg2jV\nqjWionQAgJSUaVAoFEhOHgKVqvKuWg88EIecnC+8WWad8avwXXt6M3b+ute1XGTRu5aTYnt5qywi\nkhCNSoG4WF2Vc76/i4uN8tqUs6+p/NnnH9OyGo0GdrsdaWkfub3Np9T4zbSz1W7FsYKTbseOF57k\nFDQReUxyfFMktI1BZGgA5LLKc70JbWOQHN/U26X5jLCw8Cq35ywpMWPLli9dt/lctWotnn/+RS9W\nWLf85sjXaDFBbzG4HSsqN8BoMUEXFClwVUQkRQq5HIMTYtGvcxMYzRaEBWt4xPsn7dt3wEcfzcOl\nSxdRv34DzJkzE40bN7nhbT6VSiVKS81wOp1Vfj4qVn5z5BumCUGEJtztmDYgHGEaPjaLiDxLo1Ig\nOiKIwetGdPQdGDduIl55ZQQGDeoLmUyGHj2egNFoRHJyb0yePBHDhr2EK1cuY968D9C69YMoLCxE\n796JsNvFf9MSvznyVSvUaK1rWeWc7+9aRbWEWqH2QlVERP4rPj7BdR//3/35Np/rV2+EzWiEMiwM\nn38unYuv/CZ8AaBv054AKs/xFpUboA0IR6uolq71RETkG5x2OwqyM2E+kgdbURGUWi2C49pAlzQQ\nMoX4ZxL8KnwVcgWSYnvhqSaJMFpMCNOE8IiXiMgHFWRnwrBtq2vZdvWqazl64BBvleUxfnPO91pq\nhRq6oEgGLxGRD3JYLDAfyXM7Zj5yBA6L+G9U4pfhS0REvstmNMJWVOR+TF8Em9EocEWex/AlIiKf\nogwLg1KrdT8WoYUyLEzgijyP4UtERD5FrtEgOK6N27HguDjINeK/RadfXXBFRETioEsaCKDyHK9N\nXwRlhBbBcXGu9WLH8CUiIp8jUygQPXAIovr0d/3OVwpHvL9j+BIRkc+SazRQR0d7uwyP4zlfIiIi\ngTF8iYiIBFbr8D106BDat2+PHTt2eLIeIiIiyatV+J47dw5LlixBmzbuLwUnIiKiG6tV+Op0OqSm\npiIkpO4fw/fddycwZsxIAMD06ZOxdGk6AKBjx7a4cuVynW+fiIjI02p1tXNgYGCN3xMREQSlsuZP\noujcuT06d24PAAgIUKFePQ10usrQj4wMdv25rtT15wtNSv2wF98lpX6k1AsgrX7E3Eu14ZudnY3s\n7Owq60aNGoVOnTrVaEN6fel16y5duojhw5/FgAGDsWnTejidwKRJU5CRkY5Tp/Lx8MOPIDGxJ2bN\nmoasrHUoL69ASYkFBQUmAMDVq2YoFCYsWjQfZ8+ewfTpcyCXe+4aMp0uxLUtKZBSP+zFd0mpHyn1\nAkirHzH0crMvB9WGb1JSEpKSkjxa0LUMBgO02kisWrUWkyaNQ0rKeCxevAIyGdCnz9/QuvUDN33/\n9u1b8PXXBzB/fppHg5eIiKiueP0mG3a7HfHxCQCAxo2bAgDCw8MBAJGRUSgsLLzhe/Pzf0Ba2kLM\nm7ewVlPhRERE3lCrQ8WdO3di6NCh2LNnD+bOnYvnnnuu1gUoFApoNAGVxcjlCAwM+qM4uRytWz94\nw/fOmTMTMhkQGhpa6+0TEREJrVZHvl26dEGXLl08XErNpaRMw/r1a/HRR6kYPfp1b5dDRER0S0R9\nkjQm5k6MGTMOO3ZsQ17eYW+XQ0REdEtEHb4AEBYWjjfeGI+ZM99BaWmJt8shkTh69Fv07/93b5dB\nRH5K5nQ6nUJsyNOXhFdU2FFqtiIoWA2Vqua/H74VYriUvSak1M/t9nL06LeYOvVtrF690YNV1Y6U\n9gsgrX6k1AsgrX7E0Mtt/dTI1zgcDuzP/Qk/5xfCXGxBcKgG98ZG4dH4JvypEd3U0qXp2LDhc4SF\nhaFjx84AAIvFgg8/fB95eYchl8vxyCMd8NJLr0ChUODrrw9g1qxpCAwMxIABgzF//n+RkbEKDRo0\n9HInRCR2okur/bk/4fjhCzBS0eRSAAASBklEQVQXWwAA5mILjh++gP25P3m5MvJlP/98BllZK5Ge\nvgzp6ctx+vQpAMBnn63ClSuXsXz5Z/jkkxU4duwItm3Lgd1ux/Tpk/HGGxPw6aer8euv51FeXubl\nLohIKkQVvhUVdvyc7/53v2fzC1FRYRe4IhKLo0fz8OCDbaDVRkKhUODxx58AABw4sBe9evWBUqmE\nRhOA7t2fwKFDB3H+/DlYrVa0b98BANCvXzIcDoc3WyAiCRFV+Jaara4j3j8zmywoNVsFrojEori4\nGMHBwa7lkJDK34YbDHrXnyvXh0Cv18NkKq6yPioqSrhiiUjyRBW+QcFqBIdq3I4Fh2gQFKwWuCIS\ni5CQUJjNZteywaAHAGi1kTAaja71xcVGaLVa1KtXD2Vlf9yPvKjoqnDFEpHkiSp8VSoF7o11fwRy\nT2xUnV31TOJ3//2tcPz4t9Dr9bDb7cjJ+RIA8OijHbF583rY7XaUlZUhJ+cLtG/fETExd8Fms7l+\nP75u3RrIZDJvtkBEEiK6q50fjW8CoPIcr9lkQXCIBvf8/9XORDdy331/wVNP9cPzzz+N0NAwJCT0\nwJkzp9GvXzIuXryAoUMHQCaToWvXBMTHJ0Amk+H119/CjBlTEBwcguTkwZDL5QxgIvII/s73JsTw\nO7KakFI/QvdSVlaG7t074auvdlY5d+wJUtovgLT6kVIvgLT6EUMvN/udr6imna+lUikQFhHIqWaq\nM//61zPI2ZqDq+VWbNmag3vuudfjwUtE/kl0085EQrA7nfjrgGfx3/RUVCyYB3VgEHr/61XYnU4o\nOPVMRLeJ4UvkxpfnCnE+IgYPvv6ua93P/7/+ybt13iuMiCRBtNPORHXFanfge4PZ7dj3BjOsdt5s\ng4huD8OX6E9MFTYYrDa3YwarDaYK92NERLeK4Uv0JyEqJcLV7s/IhKuVCFHxbA2J39Sp/8bevbs9\n8llffLERr776kkc+y1/wXxGiP1Er5GgeHoz9VwzXjTUPD4Zawe+sJH5vv/2Ot0vwawxfIjeeuKvy\nTmrfG8wwWG0IVyvRPDzYtZ7I12zcuA6ZmStgt9sRGRmFt99+B3l5h7F//17Uq1cPR49+C41GhZSU\nGWjcuAlGjnwBf/97bzz++N/QsWNbjBs3EatXZ8JsNmPixMnYsOFznDhxDPfc0xizZs2FUqnE3r27\n8PHHC1BRUYHAwCCMH/827rvvL95uXZT4FZ7IDYVMhifv1uHV++/GmFZ349X778aTd+v4MyPySXp9\nET74YDY++GA+MjM/R6NGMVi6NB0AcPDgPvTpk4TMzLVo164dsrNXuf0Mo9GAZcuyEB/fHRMnjsPz\nz7+IVavW4syZ0/j22zzYbDZMmzYZ48ZNwqpVa9GpU2ekpv5XuCYlhuFLdBNqhRyRAWpONZNPi4jQ\nIidnF6Kj7wAAPPBAHC5evAAAuOeexmjWrDkAoEWLFrh8+Te3n9GpUxcAQJMmTdGoUQzuuutuqNVq\nxMTcicLCAiiVSmzatBX339/qum1QzXHamYhI5Ox2O9LTF2Lfvt2w2+0oLS3FnXfeBQCoV++Pu7Ip\nFArYb/BTuaCgIACAXC5HYGCga71crnA9yzo7OxNffbUJVqsVVquV9zq/DQxfIiKR2759K/bt243U\n1DSEh4djw4bPsWXLlx7dxvHjR/HppxlIS8tAgwYN8c03BzFr1nSPbsOfcC6NiEjkDIYi1K/fAOHh\n4TAaDcjN3YqysjKPbkOv1yMiIgJ33FEf5eXl+PLLzSgvL4NAz+aRHIYvEZHIJSQ8DqPRiOTk3pg8\neSKGDXsJV65cRmrqfzy2jXbt2iMqSofk5N547bWXMWDAINSrF4xJk8Z5bBv+RLSPFBSCGB5ZVRNS\n6oe9+C4p9SOlXgBp9SOGXiT5SEEiIqr03XcnMGbMSEG3abU7cLXcynud1xIvuCIiErkWLe7H3Lmp\ngmzL7nTiy3OFbm9Aw9/B3zoe+RIRiVxe3mEkJ/cWZFtfnivE/isG6K02OAHorTbsv2LAl+cKBdm+\nVDB8iYjolvBxm57D8CUiolvCx216DsOXiIhuCR+36TkMXyIiuiW/P27THT5us2b4NYWIiG4ZH7fp\nGQxfIiK6Zb8/brNHTCRMFTaEqJQ84q0Fhi8Rkci1adMWWVnrBN2mWiFHpEIt6DalhF9XiIhExmGx\nwHrlChwWi7dLoVrikS8RkUg47XYUZGfCfCQPtqIiKLVaBMe1gS5pIGQKhbfLoxpg+BIRiURBdiYM\n27a6lm1Xr7qWowcO8VZZVAucdiYiEgGHxQLzkTy3Y+YjRzgFLTIMXyIiEbAZjbAVFbkf0xfBZjQK\nXBHdDoYvEZEIKMPCoNRq3Y9FaKEMCxO4IrodDF8iIhGQazQIjmvjdiw4Lg5yjUbgiuh28IIrIiKR\n0CUNBFB5jtemL4IyQovguDjXehIPhi8RkUjIFApEDxyCqD79YTMaoQwL4xGvSDF8iYhERq7RQB0d\n7e0y6DbwnC8REZHAGL5EREQCY/gSEREJjOFLREQkMIYvERGRwBi+REREAmP4EhERCYzhS0REJDCG\nLxERkcBqdYcrm82GiRMn4ty5c7Db7Rg3bhzatm3r6drIg/bs2YklSz6GyVSCmJgYPPPMc5gw4Q1s\n2JADlUoFAJg0aRxat34QvXv3x4IF/8XBgwdgs1WgV68+eOaZ5wAAa9ZkYe3abDidTtSrVw/jx6eg\nceMm3myNiEh0ahW+69evR2BgIFatWoVTp05h/PjxWL16tadrIw+5cOFXTJ2agqysTERENMDy5Uuw\nZk0WoqOj8fXX+9GxY2dYLBYcOvQ1Ro9+AytXLsPPP/+MZcsyYbfb8fLL/0KTJvchLq4N0tIWYu3a\nTQgKqofc3G04cGAvw5eIqIZqNe3cq1cvjB8/HgCg1WphMBg8WhR51tdfH0BcXBvExsYCAJ56qh/2\n7t2N+Pju2Lo1BwDwzTcHERv7F0RF6bBv32707dsfarUagYGBSEzsiV27cqFWayCTybBp03oUFV1F\nfHwChgz5hzdbIyISpVod+f4+TQkAGRkZePLJJz1WEHme2WzC0aNHkJiYCLvdAQAIDg5G27btsGLF\nUpSVlWH37p2Ij+8OADCZzPjww7lYtGg+AKCiogLNm7eEUqnEf/+7AMuWLcHixYvQpMl9GDv2LTRp\n0tRrvRERiZHM6XQ6b/aC7OxsZGdnV1k3atQodOrUCZ9++ilyc3OxcOHCKoHsjs1mh1KpuP2KqcbW\nrl2LnTt34sMPP7xuLDk5GUOGDMHMmTOxadMmREZGYujQoXjuuefQtWvXG36m1WpFeno6du/ejczM\nzLosn4hIcqo98k1KSkJSUtJ167Ozs5Gbm4sFCxZUG7wAoNeX1q5CL9LpQlBQYPJ2GbetefM4zJ49\nB+fPn0dAQDi+++4Etmz5CqNHv44uXRLw/vtzce+9TeFwqFFQYEK7dh3w6aer0Lx5HORyOTIyFqNZ\nsxbQ6aKxZEkaUlKmQaVS4c47m8Bm2+mVvyOp7BtAWr0A0upHSr0A0upHDL3odCE3HKvVtPP58+eR\nmZmJFStWQMMHOfu8qKgovPnmRLz88ssoL7cgKCgIr7wyFgAQH98d8+Z94LqaGQD69h2AS5cuYejQ\nAXA6nWjWrAUGDBiMwMBANGzYEEOHDoBSqUJQUBDGjHnTW20REYlWtdPO7sydOxebN29Gw4YNXesW\nL14MtVp9w/f4+jcUd8Twzaom3PVjtVqRlPR3LF/+GUJDw6r9jIoKO0rNVgQFq6FSee80gpT2jZR6\nAaTVj5R6AaTVjxh68fiR75gxYzBmzJhaF0S+IytrJdq371ht8DocDuzP/Qk/5xfCXGxBcKgG98ZG\n4dH4JpDLea8WIqKaqFX4kjQMHtwPERFaTJ8+p9rX7s/9CccPX3Atm4struWOCffVWY1ERFLE8PVj\nK1euuaXXVVTY8XN+oduxs/mFaNe5sVenoImIxIbzhVStUrMV5mKL2zGzyYJSs1XgioiIxI3hS9UK\nClYjONT9Ve3BIRoEBd/4QjsiIroew5eqpVIpcG9slNuxe2KjOOVMRFRDPOdLt+TR+MqHJ5zNL4TZ\nZEFwiAb3/P/VzkREVDMMX7olcrkcHRPuQ7vOjX3id75ERGLG8KUaUakUCIsI9HYZRESixnO+RERE\nAmP4EhERCYzhS0REJDCGLxERkcAYvkRERAJj+BIREQmM4UtERCQwhi8REZHAGL5EREQCY/gSEREJ\nTOZ0Op3eLoKIiMif8MiXiIhIYAxfIiIigTF8iYiIBMbwJSIiEhjDl4iISGAMXyIiIoExfK9hs9nw\n5ptvYtCgQRgwYAAOHz583Ws2bNiAfv36ISkpCdnZ2V6o8tYdOnQI7du3x44dO9yOt2zZEkOHDnX9\nZ7fbBa6wZqrrRyz7pqKiAmPHjsWgQYPw9NNP4/z589e9Rgz7ZsaMGUhOTsbAgQNx7NixKmP79+9H\n//79kZycjPnz53upwpq5WT/x8fEYPHiwa39cvnzZS1Xemvz8fCQkJGDFihXXjYlx39ysH7HtGxcn\nuaxevdqZkpLidDqdzvz8fGe/fv2qjJeUlDh79OjhLC4udpaVlTl79uzp1Ov1Xqi0er/88otz+PDh\nzpdeesmZm5vr9jUPP/ywwFXVXnX9iGnfrF271jl58mSn0+l07tmzx/nqq69e9xpf3zdff/2184UX\nXnA6nU7n6dOnnQMGDKgy/sQTTzgvXrzotNvtzkGDBjlPnTrljTJvWXX9dO3a1Wk2m71RWo2VlJQ4\nn376aeekSZOcy5cvv25cbPumun7EtG+uxSPfa/Tq1Qvjx48HAGi1WhgMhirjR48eRatWrRASEoKA\ngAC0adMGeXl53ii1WjqdDqmpqQgJCfF2KR5RXT9i2jcHDhxA9+7dAQCPPvqoz9Z5MwcOHEBCQgIA\noEmTJjAajTCbzQCA8+fPIywsDA0aNIBcLkfnzp1x4MABb5ZbrZv1IzZqtRppaWmIjo6+bkyM++Zm\n/YgZw/caKpUKGo0GAJCRkYEnn3yyynhhYSG0Wq1rWavVoqCgQNAab1VgYCAUCsVNX2O1WjF27FgM\nHDgQS5YsEaiy2qmuHzHtm2trlcvlkMlksFqtVV7j6/umsLAQERERruVr/74LCgpEsy9+d7N+fpeS\nkoJBgwbhvffeg9OHbwyoVCoREBDgdkyM++Zm/fxOLPvmWkpvF+At2dnZ150XHDVqFDp16oRPP/0U\nJ0+exMKFC2/6Gb6yk2/Wy82MGzcOvXr1gkwmw9NPP422bduiVatWdVnqLaltP9fy5X1z9OjRKsvu\navXVfXMjvvL37Sl/7ueVV15Bp06dEBYWhpdffhk5OTlITEz0UnV0LbHuG78N36SkJCQlJV23Pjs7\nG7m5uViwYAFUKlWVsejoaBQWFrqWr1y5ggcffLDOa63OjXqpzqBBg1x/fuSRR5Cfn+8T/8DXph8x\n7Zu33noLBQUFaNasGSoqKuB0OqFWq6u8xlf3ze/c/X3rdDq3Y5cvX/b5KcOb9QMAvXv3dv35scce\nQ35+vij+gf8zMe6b6oh133Da+Rrnz59HZmYmUlNTXdPP13rggQdw/PhxFBcXo6SkBHl5eWjbtq0X\nKr19Z86cwdixY+F0OmGz2ZCXl4f77rvP22XVmpj2TYcOHfDVV18BAHbs2IF27dpVGRfDvunQoQNy\ncnIAACdPnkR0dDSCg4MBADExMTCbzfj1119hs9mwY8cOdOjQwZvlVutm/ZhMJjz//POuUwPffPON\nz+2PWyXGfXMzYt43fKrRNebOnYvNmzejYcOGrnWLFy/G0qVL8dBDDyEuLg5fffUVFi9e7JoO7NWr\nlxcrvrGdO3di8eLFOHPmDLRaLXQ6HT755BN8/PHHrl7mzJmDgwcPQi6XIz4+HiNGjPB22Td0K/2I\nZd/Y7XZMmjQJZ8+ehVqtxrvvvosGDRqIbt+89957OHz4MGQyGVJSUvDdd98hJCQE3bt3xzfffIP3\n3nsPANCjRw88//zzXq62ejfrJyMjA+vWrYNGo0GLFi3w9ttvQyaTebtkt06cOIFZs2bhwoULUCqV\nuOOOOxAfH4+YmBhR7pvq+hHTvrkWw5eIiEhgnHYmIiISGMOXiIhIYAxfIiIigTF8iYiIBMbwJSIi\nEhjDl4iISGAMXyIiIoExfImIiAT2f66afYe2UguxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x396 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "'''\n",
    "  code by Tae Hwan Jung(Jeff Jung) @graykode\n",
    "'''\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "# 3 Words Sentence\n",
    "sentences = [ \"i like dog\", \"i like cat\", \"i like animal\",\n",
    "              \"dog cat animal\", \"apple cat dog like\", \"dog fish milk like\",\n",
    "              \"dog cat eyes like\", \"i like apple\", \"apple i hate\",\n",
    "              \"apple i movie book music like\", \"cat dog hate\", \"cat dog like\"]\n",
    "\n",
    "word_sequence = \" \".join(sentences).split()\n",
    "word_list = \" \".join(sentences).split()\n",
    "word_list = list(set(word_list))\n",
    "word_dict = {w: i for i, w in enumerate(word_list)}\n",
    "\n",
    "# Word2Vec Parameter\n",
    "batch_size = 20\n",
    "embedding_size = 2 # To show 2 dim embedding graph\n",
    "voc_size = len(word_list)\n",
    "\n",
    "def random_batch(data, size):\n",
    "    random_inputs = []\n",
    "    random_labels = []\n",
    "    random_index = np.random.choice(range(len(data)), size, replace=False)\n",
    "\n",
    "    for i in random_index:\n",
    "        random_inputs.append(np.eye(voc_size)[data[i][0]])  # target\n",
    "        random_labels.append(np.eye(voc_size)[data[i][1]])  # context word\n",
    "\n",
    "    return random_inputs, random_labels\n",
    "\n",
    "# Make skip gram of one size window\n",
    "skip_grams = []\n",
    "for i in range(1, len(word_sequence) - 1):\n",
    "    target = word_dict[word_sequence[i]]\n",
    "    context = [word_dict[word_sequence[i - 1]], word_dict[word_sequence[i + 1]]]\n",
    "\n",
    "    for w in context:\n",
    "        skip_grams.append([target, w])\n",
    "\n",
    "# Model\n",
    "inputs = tf.placeholder(tf.float32, shape=[None, voc_size])\n",
    "labels = tf.placeholder(tf.float32, shape=[None, voc_size])\n",
    "\n",
    "# W and WT is not Traspose relationship\n",
    "W = tf.Variable(tf.random_uniform([voc_size, embedding_size], -1.0, 1.0))\n",
    "WT = tf.Variable(tf.random_uniform([embedding_size, voc_size], -1.0, 1.0))\n",
    "\n",
    "hidden_layer = tf.matmul(inputs, W) # [batch_size, embedding_size]\n",
    "output_layer = tf.matmul(hidden_layer, WT) # [batch_size, voc_size]\n",
    "\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=output_layer, labels=labels))\n",
    "optimizer = tf.train.AdamOptimizer(0.001).minimize(cost)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(5000):\n",
    "        batch_inputs, batch_labels = random_batch(skip_grams, batch_size)\n",
    "        _, loss = sess.run([optimizer, cost], feed_dict={inputs: batch_inputs, labels: batch_labels})\n",
    "\n",
    "        if (epoch + 1)%1000 == 0:\n",
    "            print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n",
    "\n",
    "        trained_embeddings = W.eval()\n",
    "\n",
    "for i, label in enumerate(word_list):\n",
    "    x, y = trained_embeddings[i]\n",
    "    plt.scatter(x, y)\n",
    "    plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "Word2Vec-Tensor(Softmax).ipynb",
   "version": "0.3.2",
   "provenance": [],
   "collapsed_sections": []
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "accelerator": "GPU"
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
